Chapter 14 - Quantization and Local Inference

This chapter connects GGUF quantization labels to concrete local behavior: file size, served metadata, and short deterministic completions. The examples use the Qwen 3.5 4B Q4 file because it leaves room for KV cache and repeated tests on a 12 GB GPU.

from __future__ import annotations

import contextlib
import socket
import subprocess
import time
from pathlib import Path

import pandas as pd
import requests

pd.set_option("display.max_colwidth", 120)

LLAMA_ROOT = Path.home() / "tmp" / "llama-cpp"
BIN_DIR = LLAMA_ROOT / "llama.cpp" / "build" / "bin"
MODEL_DIR = LLAMA_ROOT / "models"
SERVER = BIN_DIR / "llama-server"
BENCH = BIN_DIR / "llama-bench"
QWEN_BF16 = MODEL_DIR / "Qwen_Qwen3.5-4B-BF16.gguf"
QWEN_Q4 = MODEL_DIR / "Qwen_Qwen3.5-4B-Q4_K_M.gguf"

def gib(path: Path) -> float:
    return path.stat().st_size / 1024**3

size_table = pd.DataFrame([
    {"file": QWEN_BF16.name, "format": "BF16", "size_gib": round(gib(QWEN_BF16), 2)},
    {"file": QWEN_Q4.name, "format": "Q4_K_M", "size_gib": round(gib(QWEN_Q4), 2)},
])
compression = gib(QWEN_BF16) / gib(QWEN_Q4)
print(f"BF16-to-Q4_K_M file-size ratio: {compression:.2f}x")
size_table

BF16-to-Q4_K_M file-size ratio: 2.80x

	file	format	size_gib
0	Qwen_Qwen3.5-4B-BF16.gguf	BF16	7.85
1	Qwen_Qwen3.5-4B-Q4_K_M.gguf	Q4_K_M	2.81

def open_port() -> int:
    with socket.socket() as sock:
        sock.bind(("127.0.0.1", 0))
        return sock.getsockname()[1]

def build_server_command(model: Path, alias: str, context: int, port: int, gpu_layers: str, flash_attn: str) -> list[str]:
    return [
        str(SERVER),
        "-m", str(model),
        "-ngl", gpu_layers,
        "-fa", flash_attn,
        "-c", str(context),
        "-np", "1",
        "--cache-type-k", "q8_0",
        "--cache-type-v", "q8_0",
        "--jinja",
        "--alias", alias,
        "--host", "127.0.0.1",
        "--port", str(port),
        "--no-webui",
        "--log-disable",
    ]

@contextlib.contextmanager
def llama_server(model: Path, alias: str = "qwen-local", context: int = 1024):
    attempts = [("full_gpu", "all", "on"), ("cpu_only", "0", "on")]
    errors = []
    proc = None
    try:
        for mode, gpu_layers, flash_attn in attempts:
            port = open_port()
            command = build_server_command(model, alias, context, port, gpu_layers, flash_attn)
            proc = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True)
            base_url = f"http://127.0.0.1:{port}"
            deadline = time.time() + 90
            ready = False
            while time.time() < deadline:
                if proc.poll() is not None:
                    errors.append(f"{mode} exited with code {proc.returncode}")
                    break
                try:
                    response = requests.get(f"{base_url}/v1/models", timeout=2)
                    if response.ok:
                        ready = True
                        break
                except requests.RequestException:
                    time.sleep(1)
            if ready:
                print(f"server_mode: {mode}")
                yield base_url, command, mode
                return
            if proc.poll() is None:
                proc.terminate()
                try:
                    proc.wait(timeout=15)
                except subprocess.TimeoutExpired:
                    proc.kill()
                    proc.wait(timeout=15)
            proc = None
        raise RuntimeError("llama-server did not become ready; " + "; ".join(errors))
    finally:
        if proc is not None and proc.poll() is None:
            proc.terminate()
            try:
                proc.wait(timeout=15)
            except subprocess.TimeoutExpired:
                proc.kill()
                proc.wait(timeout=15)

with llama_server(QWEN_Q4) as (base_url, command, mode):
    model_response = requests.get(f"{base_url}/v1/models", timeout=10).json()
    model_row = model_response["data"][0]
    metadata = model_row.get("meta") or model_row.get("metadata") or {}
    selected = {
        "id": model_row.get("id"),
        "file_type": metadata.get("general.file_type") or metadata.get("ftype"),
        "parameters": metadata.get("general.parameter_count") or metadata.get("n_params"),
        "context": metadata.get("llama.context_length") or metadata.get("n_ctx"),
        "embedding_length": metadata.get("llama.embedding_length") or metadata.get("n_embd"),
    }
    print("Server command:")
    print(" ".join(command))
    pd.DataFrame([selected])

server_mode: cpu_only
Server command:
/home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-server -m /home/alal/tmp/llama-cpp/models/Qwen_Qwen3.5-4B-Q4_K_M.gguf -ngl 0 -fa on -c 1024 -np 1 --cache-type-k q8_0 --cache-type-v q8_0 --jinja --alias qwen-local --host 127.0.0.1 --port 45667 --no-webui --log-disable

prompts = [
    "Q: In one sentence, what does quantization buy us for local LLMs?\nA:",
    "Q: Name one risk of using an overly aggressive quantization.\nA:",
    "Q: Why is KV cache memory still important after quantizing weights?\nA:",
]

rows = []
with llama_server(QWEN_Q4) as (base_url, _, mode):
    for prompt in prompts:
        start = time.perf_counter()
        response = requests.post(
            f"{base_url}/completion",
            json={
                "prompt": prompt,
                "n_predict": 64,
                "temperature": 0,
                "stop": ["\nQ:"],
            },
            timeout=90,
        )
        elapsed = time.perf_counter() - start
        response.raise_for_status()
        payload = response.json()
        timings = payload.get("timings", {})
        rows.append({
            "prompt": prompt.split("\n")[0],
            "answer": payload.get("content", "").strip(),
            "elapsed_s": round(elapsed, 2),
            "predicted_tokens": timings.get("predicted_n"),
            "predicted_tokens_per_s": round(timings.get("predicted_per_second", 0.0), 2),
        })

inference_results = pd.DataFrame(rows)
inference_results

server_mode: cpu_only

	prompt	answer	elapsed_s	predicted_tokens	predicted_tokens_per_s
0	Q: In one sentence, what does quantization buy us for local LLMs?	Quantization reduces the model size and memory footprint, enabling efficient inference on devices with limited resou...	4.91	22	5.92
1	Q: Name one risk of using an overly aggressive quantization.	<think>\n\n</think>\n\nOne significant risk of using an overly aggressive quantization is **catastrophic performance...	11.70	64	5.97
2	Q: Why is KV cache memory still important after quantizing weights?	Because quantizing weights only reduces the memory footprint of the model parameters, but the KV cache is still need...	6.60	32	5.82

bench_command = [
    str(BENCH),
    "-m", str(QWEN_Q4),
    "-ngl", "all",
    "-fa", "1",
    "-c", "2048",
    "-n", "32",
]

try:
    bench = subprocess.run(bench_command, capture_output=True, text=True, timeout=45)
    print(f"llama-bench return code: {bench.returncode}")
    text = (bench.stdout or bench.stderr).strip()
    print("\n".join(text.splitlines()[:20]))
except subprocess.TimeoutExpired:
    print("llama-bench did not finish within the timeout.")

llama-bench return code: 127
/home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-bench: symbol lookup error: /home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-bench: undefined symbol: llama_memory_breakdown_print

The served Q4 model is small enough to run repeated deterministic requests comfortably. The benchmark command is included as a stack diagnostic: if it fails while llama-server works, the immediate problem is the benchmark binary or runtime linkage rather than model availability.